import numpy as np

#生成字典
def gen_dict(files):
    dict = {}
    id = 0
    for file_path in files:
        with open(file_path, 'r') as file:
            lines = file.readlines()
            for line in lines:
                if line != "\n":
                    line_words = line.replace("\n", "").split(" ")
                    word = line_words[0]
                    if dict.get(word) == None:
                        dict[word] = id
                        id += 1
    return dict

def read_data(data_path_list, tag_to_ids, dict):

    sentence_tags_list = []
    sentence_word_ids_list = []
    for data_path in data_path_list:
        with open(data_path, 'r') as file:
            lines = file.readlines()
            sentence_tags = []
            sentence_word_ids = []
            for line in lines:
                if line == "\n":
                    sentence_tags_list.append(sentence_tags)
                    sentence_word_ids_list.append(sentence_word_ids)
                    sentence_tags = []
                    sentence_word_ids = []
                    continue
                line_words = line.replace("\n","").split(" ")
                word = line_words[0]
                tag = line_words[3]

                sentence_tags.append(tag_to_ids.get(tag))
                sentence_word_ids.append(dict.get(word))

    return sentence_word_ids_list, sentence_tags_list

